source(here::here("code/load.R"))

# extract and clean data from individual meta analyses
cleaners <- list.files(here("data/meta_analyses_raw/"), pattern = "clean.R", recursive = TRUE)
cleaners <- file.path(here("data/meta_analyses_raw"), cleaners)
for (clean in cleaners) {
  suppressMessages(suppressWarnings(
    flag <- try(source(clean), silent = TRUE)
  ))
  if (class(flag) == "try-error") {
    stop(sprintf("%s failed: %s", clean, as.character(flag)))
  }
}

# warn if there are missing extractors/cleaners
folders <- list.files(here("data/meta_analyses_raw"))
cleaners <- here("data/meta_analyses_raw", folders, "clean.R")
missing <- folders[!file.exists(cleaners)]
if (length(missing) > 0) {
    warning(sprintf("Missing cleaners: %s", paste(missing, collapse = ", ")))
}

# sanity checks: do we have all the relevant columns?
validate <- function(x) {
  missing_cols <- setdiff(c("meta_id", "question", "study_id", "estimate", "std.error"), colnames(x))
  if (length(missing_cols) > 0) {
    return(sprintf("Missing columns: %s", paste(missing_cols, collapse = ", ")))
  } else {
    return(TRUE)
  }
}

df <- list()
for (f in folders) {
  tmp <- try(suppressMessages(read_csv(here(paste0("data/meta_analyses_clean/", f, ".csv")))), silent = TRUE)
  flag <- validate(tmp)
  if (!isTRUE(flag)) {
    warning(sprintf("%s: %s", f, flag))
  } else {
    df[[f]] <- tmp
  }
}

# combine data
dat <- data.table::rbindlist(df, fill = TRUE)

# extract todos
if ("todo" %in% colnames(dat)) {
    todo <- unique(na.omit(dat[, .(meta_id, todo)]))
    todo <- sprintf("## %s\n\nTODO:\n\n * %s", todo$meta_id, todo$todo)
    todo <- paste(todo, collapse = "\n\n")
    warning(todo)
}

# write to file
write_csv(dat, here("data/estimates_rb_vab_mma.csv"))

